<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width,initial-scale=1"><title>全栈进阶那些事 - 一只爱技术的程序猿，想把分享变成一种习惯！ </title><meta name="author" content="liuyuantao"><meta name="copyright" content="liuyuantao"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:type" content="website">
<meta property="og:title" content="全栈进阶那些事">
<meta property="og:url" content="https://liuyuantao.gitee.io/page/2/index.html">
<meta property="og:site_name" content="全栈进阶那些事">
<meta property="og:description" content="一个有趣、有料、有内涵的地方!">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg">
<meta property="article:author" content="liuyuantao">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://liuyuantao.gitee.io/img/zhiyao_avatar.jpg"><link rel="shortcut icon" href="/img/favicon.png"><link rel="canonical" href="https://liuyuantao.gitee.io/page/2/"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//hm.baidu.com"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.css" media="print" onload="this.media='all'"><script>var _hmt = _hmt || [];
(function() {
  var hm = document.createElement("script");
  hm.src = "https://hm.baidu.com/hm.js?d43383137144f76a37f1074576bc674e";
  var s = document.getElementsByTagName("script")[0]; 
  s.parentNode.insertBefore(hm, s);
})();
</script><script>const GLOBAL_CONFIG = { 
  root: '/',
  algolia: undefined,
  localSearch: {"path":"search.xml","languages":{"hits_empty":"找不到您查询的内容：${query}"}},
  translate: {"defaultEncoding":1,"translateDelay":0,"msgToTraditionalChinese":"繁","msgToSimplifiedChinese":"简"},
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":200},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: true,
    post: true
  },
  runtime: '天',
  date_suffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: {"limitCount":500,"languages":{"author":"作者: liuyuantao","link":"链接: ","source":"来源: 全栈进阶那些事","info":"著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。"}},
  lightbox: 'mediumZoom',
  Snackbar: {"chs_to_cht":"你已切换为繁体","cht_to_chs":"你已切换为简体","day_to_night":"你已切换为深色模式","night_to_day":"你已切换为浅色模式","bgLight":"#49b1f5","bgDark":"#2d3035","position":"top-right"},
  source: {
    jQuery: 'https://cdn.jsdelivr.net/npm/jquery@latest/dist/jquery.min.js',
    justifiedGallery: {
      js: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/js/jquery.justifiedGallery.min.js',
      css: 'https://cdn.jsdelivr.net/npm/justifiedGallery/dist/css/justifiedGallery.min.css'
    },
    fancybox: {
      js: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.js',
      css: 'https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@latest/dist/jquery.fancybox.min.css'
    }
  },
  isPhotoFigcaption: true,
  islazyload: false,
  isanchor: true
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: '全栈进阶那些事',
  isPost: false,
  isHome: true,
  isHighlightShrink: false,
  isToc: false,
  postUpdate: '2021-11-23 18:55:57'
}</script><noscript><style type="text/css">
  #nav {
    opacity: 1
  }
  .justified-gallery img {
    opacity: 1
  }

  #recent-posts time,
  #post-meta time {
    display: inline !important
  }
</style></noscript><script>(win=>{
    win.saveToLocal = {
      set: function setWithExpiry(key, value, ttl) {
        if (ttl === 0) return
        const now = new Date()
        const expiryDay = ttl * 86400000
        const item = {
          value: value,
          expiry: now.getTime() + expiryDay,
        }
        localStorage.setItem(key, JSON.stringify(item))
      },

      get: function getWithExpiry(key) {
        const itemStr = localStorage.getItem(key)

        if (!itemStr) {
          return undefined
        }
        const item = JSON.parse(itemStr)
        const now = new Date()

        if (now.getTime() > item.expiry) {
          localStorage.removeItem(key)
          return undefined
        }
        return item.value
      }
    }
  
    win.getScript = url => new Promise((resolve, reject) => {
      const script = document.createElement('script')
      script.src = url
      script.async = true
      script.onerror = reject
      script.onload = script.onreadystatechange = function() {
        const loadState = this.readyState
        if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
        script.onload = script.onreadystatechange = null
        resolve()
      }
      document.head.appendChild(script)
    })
  
      win.activateDarkMode = function () {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = function () {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
          if (t === 'dark') activateDarkMode()
          else if (t === 'light') activateLightMode()
        
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
    const fontSizeVal = saveToLocal.get('global-font-size')
    if (fontSizeVal !== undefined) {
      document.documentElement.style.setProperty('--global-font-size', fontSizeVal + 'px')
    }
    
    const detectApple = () => {
      if (GLOBAL_CONFIG_SITE.isHome && /iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
        document.documentElement.classList.add('apple')
      }
    }
    detectApple()
    })(window)</script><meta name="generator" content="Hexo 5.4.0"><link rel="alternate" href="/atom.xml" title="全栈进阶那些事" type="application/atom+xml">
</head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/img/zhiyao_avatar.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="site-data"><div class="data-item is-center"><div class="data-item-link"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div></div><div class="data-item is-center"><div class="data-item-link"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><hr/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="page" id="body-wrap"><header class="not-top-img" id="page-header"><nav id="nav"><span id="blog_name"><a id="site-name" href="/">全栈进阶那些事</a></span><div id="menus"><div id="search-button"><a class="site-page social-icon search"><i class="fas fa-search fa-fw"></i><span> 搜索</span></a></div><div class="menus_items"><div class="menus_item"><a class="site-page" href="/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/link/"><i class="fa-fw fas fa-link"></i><span> 友链</span></a></div><div class="menus_item"><a class="site-page" href="/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page"><i class="fas fa-bars fa-fw"></i></a></div></div></nav></header><main class="layout" id="content-inner"><div class="recent-posts" id="recent-posts"><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2021/07/12/1e41bdf3e092.html" title="一文彻底搞懂Zookeeper">     <img class="post_bg" src="https://img-blog.csdnimg.cn/20200518143448459.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="一文彻底搞懂Zookeeper"></a></div><div class="recent-post-info"><a class="article-title" href="/2021/07/12/1e41bdf3e092.html" title="一文彻底搞懂Zookeeper">一文彻底搞懂Zookeeper</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2021-07-12T08:51:41.000Z" title="发表于 2021-07-12 16:51:41">2021-07-12</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/">中间件</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Zookeeper/">Zookeeper</a></span></div><div class="content">
本文是基于CentOS 7.9系统环境，进行Zookeeper的学习和使用

1. Zookeeper简介1.1 什么是ZookeeperZookeeper是一个开源的分布式的，为分布式应用提供协调服务的Apache项目。本质上，就是文件系统+通知机制
1.2 Zookeeper工作机制Zookeeper从设计模式角度来理解：是一个基于观察者模式设计的分布式服务管理框架，它负责存储和管理大家都关心的数据，然后接受观察者的注册，一旦这些数据的状态发生变化，Zookeeper就负责通知已经在Zookeeper上注册的那些观察者做出相应的反应

1.3 Zookeeper特点

Zookeeper：一各领导者（Leader），多个跟随者（Follower）组成的集群
集群中只要有半数以上节点存活，Zookeeper集群就能正常服务，所以zookeeper适合安装奇数台服务器
全局数据一致：每个server保存一份相同的数据副本，client无论连接到哪个server，数据都是一致的
更新请求顺序进行，来自同一个client的更新请求按其发送顺序依次执行
数据更新原子性，一次数据更新要么成功 ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/fd53f751c2eb.html" title="Hadoop3.X分布式运行环境搭建手记">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/c646ab2c9196c3eacc49b81b10d47f7a191.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop3.X分布式运行环境搭建手记"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/fd53f751c2eb.html" title="Hadoop3.X分布式运行环境搭建手记">Hadoop3.X分布式运行环境搭建手记</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:33:37.000Z" title="发表于 2020-11-07 13:33:37">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">1.准备虚拟机（关闭防火墙、静态ip、主机名称）1.1首先使用VMware安装CentOS7.4虚拟机安装好虚拟机后设置好静态IP(192.168.126.120,设置DNS)，hostname，执行yum update更新软件


这里由于进行了规划，所以提前设置了hosts，防止ping hostname的时候不同

1.2安装软件安装vim，在/usr/lib/jvm目录安装好jdk1.8.0_212，并配置/etc/profile文件

1.3关闭防火墙1.4创建hadoop用户创建hadoop用户并配置hadoop用户具有root权限
在根目录”/“创建/data_disk/hadoop/tmp用于存储hadoop的数据
mkdir -P /data/hadoop/tmpmkdir -P /data_disk/hadoop/datamkdir -P /data_disk/hadoop/logs

2.克隆虚拟机2.1克隆两台虚拟机2.2更改克隆虚拟机的静态IP、hostname虚拟机2的IP：192.168.126.122，hostname：node102
虚拟机3的IP：1 ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/a77fdbeecf9e.html" title="Hadoop分布式环境高可用配置">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/8310dbed7d375aa31fd2cd1111458b5e854.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop分布式环境高可用配置"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/a77fdbeecf9e.html" title="Hadoop分布式环境高可用配置">Hadoop分布式环境高可用配置</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:33:08.000Z" title="发表于 2020-11-07 13:33:08">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">
前面文章介绍过Hadoop分布式的配置，但是涉及到高可用，这次使用zookeeper配置Hadoop高可用。

1.环境准备1）修改IP2）修改主机名及主机名和IP地址的映射3）关闭防火墙4）ssh免密登录5）创建hadoop用户和用户组6）安装更新安装源、JDK、配置环境变量等
2.服务器规划


Node1
Node2
Node3



NameNode
NameNode



JournalNode
JournalNode
JournalNode


DataNode
DataNode
DataNode


ZK
ZK
ZK


ResourceManager

ResourceManager


NodeManager
NodeManager
NodeManager


3.配置Zookeeper集群参考我的之前的文章Zookeeper安装和配置说明
4.安装Hadoop1）官方下载地址：http://hadoop.apache.org/2）解压hadoop2.7.2至/usr/local/hadoop2.73）修改hadoop2.7的所属组和所属者为hadoop
chown ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/2481fd16d2ba.html" title="Hadoop架构原理简介">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/723619a14974403bb87e1981ebf7eeb8.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop架构原理简介"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/2481fd16d2ba.html" title="Hadoop架构原理简介">Hadoop架构原理简介</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:32:32.000Z" title="发表于 2020-11-07 13:32:32">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a></span></div><div class="content">一、概念Hadoop诞生于2006年，是一款支持数据密集型分布式应用并以Apache 2.0许可协议发布的开源软件框架。它支持在商品硬件构建的大型集群上运行的应用程序。Hadoop是根据Google公司发表的MapReduce和Google档案系统的论文自行实作而成。
Hadoop与Google一样，都是小孩命名的，是一个虚构的名字，没有特别的含义。从计算机专业的角度看，Hadoop是一个分布式系统基础架构，由Apache基金会开发。Hadoop的主要目标是对分布式环境下的“大数据”以一种可靠、高效、可伸缩的方式处理。
Hadoop框架透明地为应用提供可靠性和数据移动。它实现了名为MapReduce的编程范式：应用程序被分割成许多小部分，而每个部分都能在集群中的任意节点上执行或重新执行。
Hadoop还提供了分布式文件系统，用以存储所有计算节点的数据，这为整个集群带来了非常高的带宽。MapReduce和分布式文件系统的设计，使得整个框架能够自动处理节点故障。它使应用程序与成千上万的独立计算的电脑和PB级的数据。

二、组成1.Hadoop的核心组件

分析：Hadoop的核心组件分为： ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/2bd2be228806.html" title="Hadoop集群常用脚本">     <img class="post_bg" src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Hadoop集群常用脚本"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/2bd2be228806.html" title="Hadoop集群常用脚本">Hadoop集群常用脚本</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:31:44.000Z" title="发表于 2020-11-07 13:31:44">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/%E8%84%9A%E6%9C%AC/">脚本</a></span></div><div class="content">hadoop群起脚本#!/bin/bashif [ $# -lt 1 ]then    echo &quot;No Args Input...&quot;    exit ;ficase $1 in&quot;start&quot;)        echo &quot; =================== 启动 hadoop集群 ===================&quot;        echo &quot; --------------- 启动 hdfs ---------------&quot;        ssh node101 &quot;/opt/module/hadoop3.1.4/sbin/start-dfs.sh&quot;        echo &quot; --------------- 启动 yarn ---------------&quot;        ssh node102 &quot;/opt/module/hadoop3.1.4/sbin/start-yarn.sh&quot;        echo &quot; --------- ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/af7eb1570222.html" title="MapTask工作机制">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-34c625af0d544d0ceb3d92511a1e0c918bc.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MapTask工作机制"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/af7eb1570222.html" title="MapTask工作机制">MapTask工作机制</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:31:09.000Z" title="发表于 2020-11-07 13:31:09">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Yarn/">Yarn</a></span></div><div class="content">
（1）Read阶段：MapTask通过InputFormat获得的RecordReader，从输入InputSplit中解析出一个个key/value。
（2）Map****阶段：该节点主要是将解析出的key/value交给用户编写map()函数处理，并产生一系列新的key/value。
（3）Collect****收集阶段：在用户编写map()函数中，当数据处理完成后，一般会调用OutputCollector.collect()输出结果。在该函数内部，它会将生成的key/value分区（调用Partitioner），并写入一个环形内存缓冲区中。
（4）Spill****阶段：即“溢写”，当环形缓冲区满后，MapReduce会将数据写到本地磁盘上，生成一个临时文件。需要注意的是，将数据写入本地磁盘之前，先要对数据进行一次本地排序，并在必要时对数据进行合并、压缩等操作。
溢写阶段详情：
   步骤1：利用快速排序算法对缓存区内的数据进行排序，排序方式是，先按照分区编号Partition进行排序，然后按照key进行排序。这样，经过排序后，数据以分区为单位聚集在一起，且同一分区内所有数据按 ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/ecfe110a7276.html" title="ReduceTask工作机制">     <img class="post_bg" src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="ReduceTask工作机制"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/ecfe110a7276.html" title="ReduceTask工作机制">ReduceTask工作机制</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:30:17.000Z" title="发表于 2020-11-07 13:30:17">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Yarn/">Yarn</a></span></div><div class="content">
（1）Copy阶段：ReduceTask从各个MapTask上远程拷贝一片数据，并针对某一片数据，如果其大小超过一定阈值，则写到磁盘上，否则直接放到内存中。
（2）Sort****阶段：在远程拷贝数据的同时，ReduceTask启动了两个后台线程对内存和磁盘上的文件进行合并，以防止内存使用过多或磁盘上文件过多。按照MapReduce语义，用户编写reduce()函数输入数据是按key进行聚集的一组数据。为了将key相同的数据聚在一起，Hadoop采用了基于排序的策略。由于各个MapTask已经实现对自己的处理结果进行了局部排序，因此，ReduceTask只需对所有数据进行一次归并排序即可。
（3）Reduce阶段：reduce()函数将计算结果写到HDFS上。
</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/947729c8efa7.html" title="Yarn工作机制">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-70c2f401ba4415958505e78cca120abc427.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Yarn工作机制"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/947729c8efa7.html" title="Yarn工作机制">Yarn工作机制</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:29:43.000Z" title="发表于 2020-11-07 13:29:43">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Yarn/">Yarn</a></span></div><div class="content">
（1）MR程序提交到客户端所在的节点。（2）YarnRunner向ResourceManager申请一个Application。（3）RM将该应用程序的资源路径返回给YarnRunner。（4）该程序将运行所需资源提交到HDFS上。（5）程序资源提交完毕后，申请运行mrAppMaster。（6）RM将用户的请求初始化成一个Task。（7）其中一个NodeManager领取到Task任务。（8）该NodeManager创建容器Container，并产生MRAppmaster。（9）Container从HDFS上拷贝资源到本地。（10）MRAppmaster向RM 申请运行MapTask资源。（11）RM将运行MapTask任务分配给另外两个NodeManager，另两个NodeManager分别领取任务并创建容器。（12）MR向两个接收到任务的NodeManager发送程序启动脚本，这两个NodeManager分别启动MapTask，MapTask对数据分区排序。（13）MrAppMaster等待所有MapTask运行完毕后，向RM申请容器，运行ReduceTask。（14）Reduc ...</div></div></div><div class="recent-post-item"><div class="post_cover left_radius"><a href="/2020/11/07/8bc5bf1e47da.html" title="作业提交过程之HDFS">     <img class="post_bg" src="https://oscimg.oschina.net/oscnet/up-5e59cb5a26ed324a28f500d70fe4e642254.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="作业提交过程之HDFS"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/8bc5bf1e47da.html" title="作业提交过程之HDFS">作业提交过程之HDFS</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:28:54.000Z" title="发表于 2020-11-07 13:28:54">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/HDFS/">HDFS</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Yarn/">Yarn</a></span></div><div class="content">
作业提交全过程详解（1）作业提交第1步：Client调用job.waitForCompletion方法，向整个集群提交MapReduce作业。第2步：Client向RM申请一个作业id。第3步：RM给Client返回该job资源的提交路径和作业id。第4步：Client提交jar包、切片信息和配置文件到指定的资源提交路径。第5步：Client提交完资源后，向RM申请运行MrAppMaster。
（2）作业初始化第6步：当RM收到Client的请求后，将该job添加到容量调度器中。第7步：某一个空闲的NM领取到该Job。第8步：该NM创建Container，并产生MRAppmaster。第9步：下载Client提交的资源到本地。
（3）任务分配第10步：MrAppMaster向RM申请运行多个MapTask任务资源。第11步：RM将运行MapTask任务分配给另外两个NodeManager，另两个NodeManager分别领取任务并创建容器。
（4）任务运行第12步：MR向两个接收到任务的NodeManager发送程序启动脚本，这两个NodeManager分别启动MapTask，Map ...</div></div></div><div class="recent-post-item"><div class="post_cover right_radius"><a href="/2020/11/07/5add4ff74271.html" title="MapReduce开发总结">     <img class="post_bg" src="https://cdn.pixabay.com/photo/2021/08/21/23/14/book-shop-6563786__340.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MapReduce开发总结"></a></div><div class="recent-post-info"><a class="article-title" href="/2020/11/07/5add4ff74271.html" title="MapReduce开发总结">MapReduce开发总结</a><div class="article-meta-wrap"><span class="post-meta-date"><i class="far fa-calendar-alt"></i><span class="article-meta-label">发表于</span><time datetime="2020-11-07T05:28:21.000Z" title="发表于 2020-11-07 13:28:21">2020-11-07</time></span><span class="article-meta"><span class="article-meta__separator">|</span><i class="fas fa-inbox"></i><a class="article-meta__categories" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/">大数据</a></span><span class="article-meta tags"><span class="article-meta__separator">|</span><i class="fas fa-tag"></i><a class="article-meta__tags" href="/tags/Hadoop/">Hadoop</a><span class="article-meta__link">•</span><a class="article-meta__tags" href="/tags/Yarn/">Yarn</a></span></div><div class="content">1****）输入数据接口：InputFormat（1）默认使用的实现类是：TextInputFormat
（2）TextInputFormat的功能逻辑是：一次读一行文本，然后将该行的起始偏移量作为key，行内容作为value返回。
（3）CombineTextInputFormat可以把多个小文件合并成一个切片处理，提高处理效率。
2****）逻辑处理接口：Mapper用户根据业务需求实现其中三个方法：map()   setup()   cleanup ()
3）Partitioner分区（1）有默认实现 HashPartitioner，逻辑是根据key的哈希值和numReduces来返回一个分区号；key.hashCode()&amp;Integer.MAXVALUE % numReduces
（2）如果业务上有特别的需求，可以自定义分区。
4）Comparable排序（1）当我们用自定义的对象作为key来输出时，就必须要实现WritableComparable接口，重写其中的compareTo()方法。
（2）部分排序：对最终输出的每一个文件进行内部排序。
（3）全排序：对所有数 ...</div></div></div><nav id="pagination"><div class="pagination"><a class="extend prev" rel="prev" href="/"><i class="fas fa-chevron-left fa-fw"></i></a><a class="page-number" href="/">1</a><span class="page-number current">2</span><a class="page-number" href="/page/3/#content-inner">3</a><span class="space">&hellip;</span><a class="page-number" href="/page/5/#content-inner">5</a><a class="extend next" rel="next" href="/page/3/#content-inner"><i class="fas fa-chevron-right fa-fw"></i></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/img/zhiyao_avatar.jpg" onerror="this.onerror=null;this.src='/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">liuyuantao</div><div class="author-info__description">一个有趣、有料、有内涵的地方!</div></div><div class="card-info-data"><div class="card-info-data-item is-center"><a href="/archives/"><div class="headline">文章</div><div class="length-num">41</div></a></div><div class="card-info-data-item is-center"><a href="/tags/"><div class="headline">标签</div><div class="length-num">14</div></a></div><div class="card-info-data-item is-center"><a href="/categories/"><div class="headline">分类</div><div class="length-num">4</div></a></div></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn card-announcement-animation"></i><span>公告</span></div><div class="announcement_content"><p><strong style="color:#858585;font-size:20px;">扫描二维码,关注我们</strong><br/><span style="color:#49b1f5;">了解更多全栈技能!</span></p><img src="/img/qrcode_for_gh_da91cebd00cc_258.jpg"></div></div><div class="sticky_layout"><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割"><img src="https://cdn.pixabay.com/photo/2021/10/25/17/16/nature-6741602__340.jpg" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="Nginx日志按天自动切割"/></a><div class="content"><a class="title" href="/2021/11/19/4e544b17c3ac.html" title="Nginx日志按天自动切割">Nginx日志按天自动切割</a><time datetime="2021-11-19T09:40:06.000Z" title="发表于 2021-11-19 17:40:06">2021-11-19</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置"><img src="https://cdn.pixabay.com/photo/2017/01/31/18/21/books-2026194__340.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="ClickHouse新特性之SQL化用户配置"/></a><div class="content"><a class="title" href="/2021/11/18/b4485f4970ce.html" title="ClickHouse新特性之SQL化用户配置">ClickHouse新特性之SQL化用户配置</a><time datetime="2021-11-18T05:55:38.000Z" title="发表于 2021-11-18 13:55:38">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度"><img src="https://oscimg.oschina.net/oscnet/up-8479e16ac3738d1e8243838c11c9e96aba9.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="MergeTree引擎的固定/自适应索引粒度"/></a><div class="content"><a class="title" href="/2021/11/18/6d7187a575ff.html" title="MergeTree引擎的固定/自适应索引粒度">MergeTree引擎的固定/自适应索引粒度</a><time datetime="2021-11-18T05:52:18.000Z" title="发表于 2021-11-18 13:52:18">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法"><img src="https://oscimg.oschina.net/oscnet/up-ac406843f42bfac15c16815aeb376bca9ad.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="无缝更改ClickHouse物化视图SELECT逻辑的方法"/></a><div class="content"><a class="title" href="/2021/11/18/23f3f36da036.html" title="无缝更改ClickHouse物化视图SELECT逻辑的方法">无缝更改ClickHouse物化视图SELECT逻辑的方法</a><time datetime="2021-11-18T05:43:50.000Z" title="发表于 2021-11-18 13:43:50">2021-11-18</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例"><img src="https://upload-images.jianshu.io/upload_images/195230-462138b5b8e6e815.png" onerror="this.onerror=null;this.src='/img/404.jpg'" alt="物化视图简介与ClickHouse中的应用示例"/></a><div class="content"><a class="title" href="/2021/11/18/fec2d399fcb4.html" title="物化视图简介与ClickHouse中的应用示例">物化视图简介与ClickHouse中的应用示例</a><time datetime="2021-11-18T05:31:13.000Z" title="发表于 2021-11-18 13:31:13">2021-11-18</time></div></div></div></div><div class="card-widget card-categories"><div class="item-headline">
            <i class="fas fa-folder-open"></i>
            <span>分类</span>
            
            </div>
            <ul class="card-category-list" id="aside-cat-list">
            <li class="card-category-list-item "><a class="card-category-list-link" href="/categories/Spring-SpringBoot/"><span class="card-category-list-name">Spring&SpringBoot</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E4%B8%AD%E9%97%B4%E4%BB%B6/"><span class="card-category-list-name">中间件</span><span class="card-category-list-count">2</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E5%A4%A7%E6%95%B0%E6%8D%AE/"><span class="card-category-list-name">大数据</span><span class="card-category-list-count">34</span></a></li><li class="card-category-list-item "><a class="card-category-list-link" href="/categories/%E8%BF%90%E7%BB%B4/"><span class="card-category-list-name">运维</span><span class="card-category-list-count">1</span></a></li>
            </ul></div><div class="card-widget card-tags"><div class="item-headline"><i class="fas fa-tags"></i><span>标签</span></div><div class="card-tag-cloud"><a href="/tags/Hadoop/" style="font-size: 1.45em; color: rgb(29, 18, 195)">Hadoop</a><a href="/tags/HDFS/" style="font-size: 1.39em; color: rgb(67, 37, 146)">HDFS</a><a href="/tags/ClickHouse/" style="font-size: 1.27em; color: rgb(1, 73, 96)">ClickHouse</a><a href="/tags/Yarn/" style="font-size: 1.33em; color: rgb(150, 127, 36)">Yarn</a><a href="/tags/%E8%84%9A%E6%9C%AC/" style="font-size: 1.15em; color: rgb(88, 131, 138)">脚本</a><a href="/tags/Nginx/" style="font-size: 1.15em; color: rgb(178, 37, 116)">Nginx</a><a href="/tags/%E8%BF%90%E7%BB%B4/" style="font-size: 1.15em; color: rgb(128, 60, 172)">运维</a><a href="/tags/HBase/" style="font-size: 1.15em; color: rgb(0, 104, 140)">HBase</a><a href="/tags/%E4%B8%AD%E9%97%B4%E4%BB%B6/" style="font-size: 1.21em; color: rgb(40, 97, 23)">中间件</a><a href="/tags/Zookeeper/" style="font-size: 1.15em; color: rgb(40, 178, 154)">Zookeeper</a><a href="/tags/Hive/" style="font-size: 1.27em; color: rgb(147, 104, 22)">Hive</a><a href="/tags/Kafka/" style="font-size: 1.15em; color: rgb(176, 132, 95)">Kafka</a><a href="/tags/MapReduce/" style="font-size: 1.15em; color: rgb(157, 129, 39)">MapReduce</a><a href="/tags/Flume/" style="font-size: 1.15em; color: rgb(72, 148, 10)">Flume</a></div></div><div class="card-widget card-archives"><div class="item-headline"><i class="fas fa-archive"></i><span>归档</span></div><ul class="card-archive-list"><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/11/"><span class="card-archive-list-date">十一月 2021</span><span class="card-archive-list-count">8</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/10/"><span class="card-archive-list-date">十月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/09/"><span class="card-archive-list-date">九月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2021/07/"><span class="card-archive-list-date">七月 2021</span><span class="card-archive-list-count">1</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/11/"><span class="card-archive-list-date">十一月 2020</span><span class="card-archive-list-count">27</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2020/09/"><span class="card-archive-list-date">九月 2020</span><span class="card-archive-list-count">2</span></a></li><li class="card-archive-list-item"><a class="card-archive-list-link" href="/archives/2019/11/"><span class="card-archive-list-date">十一月 2019</span><span class="card-archive-list-count">1</span></a></li></ul></div><div class="card-widget card-webinfo"><div class="item-headline"><i class="fas fa-chart-line"></i><span>网站资讯</span></div><div class="webinfo"><div class="webinfo-item"><div class="item-name">文章数目 :</div><div class="item-count">41</div></div><div class="webinfo-item"><div class="item-name">已运行时间 :</div><div class="item-count" id="runtimeshow" data-publishDate="2020-08-31T16:00:00.000Z"></div></div><div class="webinfo-item"><div class="item-name">本站总字数 :</div><div class="item-count">59.5k</div></div><div class="webinfo-item"><div class="item-name">本站访客数 :</div><div class="item-count" id="busuanzi_value_site_uv"></div></div><div class="webinfo-item"><div class="item-name">本站总访问量 :</div><div class="item-count" id="busuanzi_value_site_pv"></div></div><div class="webinfo-item"><div class="item-name">最后更新时间 :</div><div class="item-count" id="last-push-date" data-lastPushDate="2021-11-23T10:55:57.650Z"></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2021 By liuyuantao</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener external nofollow noreferrer" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div><div class="footer_custom_text">扫描二维码,关注我们!了解更多全栈技能!<br/><img style="width:100px;height:100px;" src="/img/qrcode_for_gh_da91cebd00cc_258.jpg" ></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="font-plus" type="button" title="放大字体"><i class="fas fa-plus"></i></button><button id="font-minus" type="button" title="缩小字体"><i class="fas fa-minus"></i></button><button id="translateLink" type="button" title="简繁转换">简</button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside_config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button id="go-up" type="button" title="回到顶部"><i class="fas fa-arrow-up"></i></button></div></div><div id="local-search"><div class="search-dialog"><div class="search-dialog__title" id="local-search-title">本地搜索</div><div id="local-input-panel"><div id="local-search-input"><div class="local-search-box"><input class="local-search-box--input" placeholder="搜索文章" type="text"/></div></div></div><hr/><div id="local-search-results"></div><span class="search-close-button"><i class="fas fa-times"></i></span></div><div id="search-mask"></div></div><div><script src="/js/utils.js"></script><script src="/js/main.js"></script><script src="/js/tw_cn.js"></script><script src="https://cdn.jsdelivr.net/npm/medium-zoom/dist/medium-zoom.min.js"></script><script src="https://cdn.jsdelivr.net/npm/instant.page/instantpage.min.js" type="module"></script><script src="https://cdn.jsdelivr.net/npm/node-snackbar/dist/snackbar.min.js"></script><script src="/js/search/local-search.js"></script><script>var preloader = {
  endLoading: () => {
    document.body.style.overflow = 'auto';
    document.getElementById('loading-box').classList.add("loaded")
  },
  initLoading: () => {
    document.body.style.overflow = '';
    document.getElementById('loading-box').classList.remove("loaded")

  }
}
window.addEventListener('load',preloader.endLoading())</script><div class="js-pjax"></div><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc@1/dist/canvas-fluttering-ribbon.min.js"></script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>